package web.parshant.crawler;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;

import org.htmlcleaner.CommentNode;
import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.HtmlNode;
import org.htmlcleaner.TagNode;
import org.htmlcleaner.TagNodeVisitor;
import org.htmlcleaner.Utils;

public class SimpleCrawler {
	public static void main(String[] args) {
		try {
			URL simpleURL = new URL("http://en.wikipedia.org");
            
            HtmlCleaner cleaner = new HtmlCleaner();
            TagNode node = cleaner.clean(simpleURL);
            
            node.traverse(new TagNodeVisitor() {
				
				@Override
				public boolean visit(TagNode tagNode, HtmlNode htmlNode) {
					if (htmlNode instanceof TagNode) {
			            TagNode tag = (TagNode) htmlNode;
			            String tagName = tag.getName();
			            if ("img".equals(tagName)) {
			                String src = tag.getAttributeByName("src");
			                if (src != null) {
			                    tag.setAttribute("src", Utils.fullUrl("http://en.wikipedia.org", src));
			                }
			            } else if ("a".equals(tagName)){
			            	System.out.println("A tag : "+tag.getAttributeByName("href"));
			            } else if ("meta".equals(tagName)){
			            	System.out.println("Meta tag : "+tag.getAttributeByName("content"));
			            }
			        } else if (htmlNode instanceof CommentNode) {
			            CommentNode comment = ((CommentNode) htmlNode);
			            comment.getContent().append(" -- By HtmlCleaner");
			        }
					return true;
				}
			});
            
		} catch (MalformedURLException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
}
